﻿
'reference to http://msdn.microsoft.com/en-sg/magazine/cc188929(en-us).aspx#S1
'http://www.thecodinghumanist.com/Content/HowToWriteAWebCrawlerInCSharp.aspx

'#################################################
'
'   clsSpider
'       This class is fired by the main form
'       of the spider as many times as is 
'       specified by the main application
'   
'       This class runs completely independently
'       
'   Entry Point:  New
'   Main Public Function:   ProcessURL
'
'#################################################

'Imports .Net which is all of our web functionality 
'and .IO which allows us to do some basic disk functions
Imports System.Net
Imports System.IO
Imports System.Data.SqlClient
Imports System
Imports System.Text.RegularExpressions
Imports System.Threading


Public Class clsSpider

    Private m_sURLToProcess As String          'The full URL to Process
    Private m_sURLProtocol As String           'The protocol of the URL (http, ftp, etc.)
    Private m_sURLBase As String               'The Base name for the site:  www.optigon.net (not including the "/")
    Private m_sURLPath As String               'The Path for the page: /images/old
    Private m_sURLPageName As String           'The page name for the page (including the "/"): /MyLogo.jpg
    Private m_sFileExtensions() As String      'The file extensions (.jpg, .gif, etc.)
    Private m_sSiteName As String              'The site name to parse

    Private m_iThreadIndex As Integer          'Which thread am I running on?

    Private m_iBytesRead As Long              'Bytes read from the web stream
    Private m_sPages() As String              'An array of pages that need to be downloaded based on links found on this page

    Private m_sPageResponded As String        'The page that responded to the URI request
    Private m_bParseWordsIntoDB As Boolean     'Parse the entries into the database
    Private m_bObeyRobots As Boolean           'See if the search should obey the robots.txt file
    Private strConn As String = My.Settings.dbConn.ToString()


    'Event denoting the current thread status for this thread
    Public Event evThreadStatus(ByVal iCurrentPos As Integer, _
                                ByVal iTotalPos As Integer, _
                                ByVal iThreadIndex As Long)

    'Event that sends the thread ID to the calling function
    Public Event evSendThreadID(ByVal iThreadID As Long, _
                                ByVal iThreadIndex As Long)

    'Event that lets the parent process know when the 
    'current function is complete
    Public Event evFinished(ByVal sPages() As String, _
                            ByVal iThreadIndex As Integer, _
                            ByVal sURLToProcess As String, _
                            ByVal sURLProtocol As String, _
                            ByVal sURLBase As String, _
                            ByVal sPageResponded As String)


    'This sub grabs information from the calling process.  This is a new feature in VB.NET
    'It's basically the same as a constructor class in C++
    Public Sub New(ByVal pm_sURLToProcess As String, _
                    ByVal pm_sURLProtocol As String, _
                    ByVal pm_sURLBase As String, _
                    ByVal pm_sURLPath As String, _
                    ByVal pm_sURLPageName As String, _
                    ByVal pm_sFileExtensions() As String, _
                    ByVal pm_sSiteName As String, _
                    ByVal pm_iThreadIndex As Integer, _
                    ByVal pm_bParseWordsIntoDB As Boolean, _
                    ByVal pm_bObeyRobots As Boolean)

        Try
            'Set the values
            m_sURLToProcess = pm_sURLToProcess
            m_sURLProtocol = pm_sURLProtocol
            m_sURLBase = pm_sURLBase
            m_sURLPath = pm_sURLPath
            m_sURLPageName = pm_sURLPageName
            m_sFileExtensions = pm_sFileExtensions
            m_sSiteName = pm_sSiteName
            m_iThreadIndex = pm_iThreadIndex
            m_bParseWordsIntoDB = pm_bParseWordsIntoDB
            m_bObeyRobots = pm_bObeyRobots
        Catch

            Debug.WriteLine("clsSpider_New: " & Err.Number & ": " & Err.Description & "  " & Err.Erl)
            WriteErrorLog("clsSpider_New: " & Err.Number & ": " & Err.Description & "  " & Err.Erl)

        End Try

    End Sub

    Public Sub ProcessURL()
        'Process the URL the user just sent
        Try
            'Send an even back to the calling function letting it know how we're doing.
            RaiseEvent evSendThreadID(AppDomain.GetCurrentThreadId, m_iThreadIndex)

            Dim i As Integer = 0
            Dim bParse As Boolean = True
            Dim bSaveToDB As Boolean = True

            Debug.WriteLine("Thread processing: " & m_sURLToProcess)      'Write a line to the debugger

            'Get the request from the web server(s)
            Dim myReq As HttpWebRequest = HttpWebRequest.Create(m_sURLToProcess)
            Dim myResponse As HttpWebResponse = myReq.GetResponse

            'Clear the page array
            ReDim m_sPages(0)

            'Get the page name that we got back
            m_sPageResponded = myResponse.ResponseUri.ToString()


            'Get the data from the stream
            Dim iContentLength As Integer
            Dim sTotalBuffer() As Byte
            If myResponse.ContentLength >= 0 Then
                '*********************  Take care of if the Content Length can be found (if the string can be seeked)
                iContentLength = myResponse.ContentLength

                Dim br As New BinaryReader(myResponse.GetResponseStream())
                Dim iTotalBytes As Integer = 0
                RaiseEvent evThreadStatus(0, iContentLength, m_iThreadIndex)

                ReDim sTotalBuffer(iContentLength - 1)
                Dim sBuffer() As Byte
                m_iBytesRead = 1
                iTotalBytes = 0

                Do Until m_iBytesRead = 0
                    ReDim sBuffer(iContentLength - 1)
                    m_iBytesRead = br.Read(sBuffer, 0, iContentLength)
                    ReDim Preserve sBuffer(m_iBytesRead - 1)
                    If m_iBytesRead > 0 Then Array.Copy(sBuffer, 0, sTotalBuffer, iTotalBytes, sBuffer.Length)
                    iTotalBytes += m_iBytesRead
                    RaiseEvent evThreadStatus(iTotalBytes, iContentLength, m_iThreadIndex)
                Loop

            Else
                '*********************  Take care of if the Content Length cannot be found (if the string can't be seeked)
                iContentLength = 1024

                Dim br As New BinaryReader(myResponse.GetResponseStream())
                Dim iTotalBytes As Integer = 0
                RaiseEvent evThreadStatus(0, iContentLength, m_iThreadIndex)

                ReDim sTotalBuffer(0)
                Dim sBuffer() As Byte
                m_iBytesRead = 1
                iTotalBytes = 0
                Do Until m_iBytesRead = 0
                    ReDim sBuffer(iContentLength - 1)
                    m_iBytesRead = br.Read(sBuffer, 0, iContentLength)
                    ReDim Preserve sBuffer(m_iBytesRead - 1)
                    ReDim Preserve sTotalBuffer((UBound(sTotalBuffer) + 1) + m_iBytesRead - 1)
                    If m_iBytesRead > 0 Then Array.Copy(sBuffer, 0, sTotalBuffer, iTotalBytes, sBuffer.Length)
                    iTotalBytes += m_iBytesRead
                    RaiseEvent evThreadStatus(iTotalBytes, iContentLength, m_iThreadIndex)
                Loop

            End If

            'Raise an event to tell that we've finished retreiving the stream from the web
            RaiseEvent evThreadStatus(iContentLength, iContentLength, m_iThreadIndex)


            'Check out if this site has a no content follow on it
            Dim sTmpHTML As String = System.Text.Encoding.ASCII.GetString(sTotalBuffer)
            If m_bObeyRobots Then
                'Check to see if there are meta inclusion tags
                If InStr(1, Replace(sTmpHTML, Chr(34), ""), "<meta name=robots content=noindex>", CompareMethod.Text) <> 0 Then
                    bSaveToDB = False
                End If
                If InStr(1, Replace(sTmpHTML, Chr(34), ""), "<meta name=robots content=nofollow>", CompareMethod.Text) <> 0 Then
                    bParse = False
                End If
            End If

            'Write the element to a database
            If bSaveToDB Then WriteToDB(sTotalBuffer, m_bParseWordsIntoDB)

            'Fire the code to get the listing of other pages
            'Only fire it for pages that need it
            If bParse Then
                'Get the stream from the file
                Dim sOut As String = sTmpHTML

                'Replace all the characters in the string
                sOut = Replace(sOut, vbCrLf, "")
                sOut = Replace(sOut, vbCr, "")
                sOut = Replace(sOut, vbLf, "")
                sOut = Replace(sOut, vbTab, "")
                ParsePageForOtherPages(sOut)
            End If

            'Close the web pipe
            myResponse.Close()

            'Close out the sub by raising an event
            RaiseEvent evFinished(m_sPages, _
                                    m_iThreadIndex, _
                                    m_sURLToProcess, _
                                    m_sURLProtocol, _
                                    m_sURLBase, _
                                    m_sPageResponded)

        Catch

            Debug.WriteLine("ProcessURL: " & m_sURLToProcess & ": " & Err.Number & ": " & Err.Description & "  " & Err.Erl)
            WriteErrorLog("ProcessURL: " & m_sURLToProcess & ": " & Err.Number & ": " & Err.Description & "  " & Err.Erl)

        End Try

    End Sub

    Private Sub ParsePageForOtherPages(ByVal sText As String)
        'Redim the variable for the pages
        Try
            'Take care of the parse tags
            ParseSubString(sText, "<a ", " href=" & Chr(34))

        Catch
            'MsgBox(Err.Number & ": " & Err.Description & "  " & Err.Erl)
            Debug.WriteLine("ParsePageForOtherPages: " & m_sURLToProcess & ": " & Err.Number & ": " & Err.Description & "  " & Err.Erl)
            WriteErrorLog("ParsePageForOtherPages: " & m_sURLToProcess & ": " & Err.Number & ": " & Err.Description & "  " & Err.Erl)

        End Try

    End Sub

    Private Sub ParseSubString(ByVal sTmp As String, _
                                        ByVal sKeyType As String, _
                                        ByVal sKeySrc As String)

        'sKeyType would be like "<a " or "<img "
        'sKeySrc would be like " href=" or " src=", respectively
        Dim sTmpPageName As String = ""
        Dim iStartPos As Integer = 0
        Dim iEndPos As Integer = 0

        Dim iEndPosQuote As Integer = 0
        Dim iEndPosSpace As Integer = 0
        Dim iEndPosCloseTag As Integer = 0

        Try

            'Need to do a sub parse for the function above
            Do Until InStr(1, sTmp, sKeyType, CompareMethod.Text) = 0
                'Find the position of the page address
                iStartPos = InStr(1, sTmp, sKeyType, CompareMethod.Text)
                iStartPos = InStr(iStartPos, sTmp, sKeySrc, CompareMethod.Text) + Len(sKeySrc)
                iEndPosQuote = InStr(iStartPos, sTmp, Chr(34), CompareMethod.Text)
                iEndPosCloseTag = InStr(iStartPos, sTmp, ">", CompareMethod.Text)
                iEndPosSpace = InStr(iStartPos, sTmp, " ", CompareMethod.Text)

                If iEndPosSpace = 0 Then iEndPosSpace = iEndPosCloseTag
                If iEndPosQuote = 0 Then iEndPosQuote = iEndPosCloseTag

                If iEndPosQuote > 0 And iEndPosQuote < iEndPosCloseTag And iEndPosQuote < iEndPosSpace Then
                    iEndPos = iEndPosQuote
                ElseIf iEndPosSpace > 0 And iEndPosSpace < iEndPosCloseTag Then
                    iEndPos = iEndPosSpace
                Else
                    iEndPos = iEndPosCloseTag
                End If
                sTmpPageName = Mid(sTmp, iStartPos, iEndPos - iStartPos)

                'Check to see if they've got an invalid type here
                Select Case True
                    Case InStr(1, sTmpPageName, "mailto:", CompareMethod.Text) <> 0     'Don't do anything - mailto
                    Case InStr(1, sTmpPageName, "javascript:", CompareMethod.Text) <> 0     'Don't do anything - javascript
                    Case InStr(1, sTmpPageName, "#", CompareMethod.Text) <> 0     'Don't do anything - # reference for anchoring
                    Case Else
                        'Check to see if they've got another URL here
                        'Add the element to the array
                        'Debug.WriteLine(sTmpPageName)
                        If Trim(m_sPages(UBound(m_sPages))) <> "" Then ReDim Preserve m_sPages(UBound(m_sPages) + 1)

                        If Left(LCase(sTmpPageName), 7) = "http://" And _
                          InStr(1, sTmpPageName, m_sURLProtocol & m_sURLBase, CompareMethod.Text) = 0 Then

                            m_sPages(UBound(m_sPages)) = sTmpPageName
                        Else
                            sTmpPageName = Replace(sTmpPageName, m_sURLProtocol, "", 1, -1, CompareMethod.Text)
                            sTmpPageName = Replace(sTmpPageName, m_sURLBase, "", 1, -1, CompareMethod.Text)
                            m_sPages(UBound(m_sPages)) = m_sURLProtocol & m_sURLBase & _
                                IIf(Left(sTmpPageName, 1) = "/", sTmpPageName, "/" & sTmpPageName)

                        End If
                End Select

                'Take the element off the string
                sTmp = Mid(sTmp, iEndPos + 1)

            Loop

        Catch
            Debug.WriteLine("ParseSubString: " & m_sURLToProcess & ": " & Err.Number & ": " & Err.Description & "  " & Err.Erl)
            WriteErrorLog("ParseSubString: " & m_sURLToProcess & ": " & Err.Number & ": " & Err.Description & "  " & Err.Erl)

        End Try

    End Sub

    Private Sub WriteToDB(ByVal sBuffer() As Byte, ByVal bParseWordsInDB As Boolean)
        'Write the element to the database
        Dim bInsideTag As Boolean = False               'Check to see if we're inside a tag or not
        Dim i As Long = 0, iPosition As Long = 0        'iPosition is the position that words appear in the text
        Dim sSQL As String = ""                         'The SQL String to hold updates to the system
        Dim sOut As String = ""                         'The actual string that we're going to be writing to the database
        Dim iPageID As Long                             'The page id that we're going to be working with        

        Try
            'Get the HTML from the passed buffer in Unicode text
            Dim sTmpHTML As String = System.Text.Encoding.ASCII.GetString(sBuffer)

            Dim filetitle As String = ""
            Dim filesize As Long = 0
            Dim filedesc As String = ""
            Dim fileKeyword As String = ""
            Dim filecontents As String = sTmpHTML

            Dim TitleMatch As Match = (Regex.Match(filecontents, "<title>([^<]*)</title>", RegexOptions.IgnoreCase Or RegexOptions.Multiline))
            filetitle = TitleMatch.Groups(0).Value
            filetitle = ReturnOutput(filetitle) 'Remove Tag 

            '// ### Parse out META data for description ###
            'Dim DescriptionMatch As Match
            'DescriptionMatch = (Regex.Match(filecontents, "<META NAME=""DESCRIPTION"" CONTENT=""([^<]*)"">", RegexOptions.IgnoreCase Or RegexOptions.Multiline))
            'filedesc = DescriptionMatch.Groups(1).Value
            'filedesc = ReturnOutput(filedesc) 'Remove Tag

            '// ### Parse out META data for keyword ###
            'Dim keywordMatch As Match
            'keywordMatch = (Regex.Match(filecontents, "<META NAME=""KEYWORDS"" CONTENT=""([^<]*)"">", RegexOptions.IgnoreCase Or RegexOptions.Multiline))
            'fileKeyword = keywordMatch.Groups(2).Value
            'fileKeyword = ReturnOutput(fileKeyword) 'Remove Tag

            Dim metaKey As String = [String].Empty, metaValue As String = [String].Empty
            For Each metamatch As Match In Regex.Matches(filecontents, "<meta\s*(?:(?:\b(\w|-)+\b\s*(?:=\s*(?:""[^""]*""|'[^']*'|[^""'<> ]+)\s*)?)*)/?\s*>", RegexOptions.IgnoreCase Or RegexOptions.ExplicitCapture)
                metaKey = [String].Empty
                metaValue = [String].Empty
                ' Loop through the attribute/value pairs inside the tag
                For Each submetamatch As Match In Regex.Matches(metamatch.Value.ToString(), "(?<name>\b(\w|-)+\b)\s*=\s*(""(?<value>[^""]*)""|'(?<value>[^']*)'|(?<value>[^""'<> ]+)\s*)+", RegexOptions.IgnoreCase Or RegexOptions.ExplicitCapture)

                    If "http-equiv" = submetamatch.Groups(1).ToString().ToLower() Then
                        metaKey = submetamatch.Groups(2).ToString()
                    End If
                    If ("name" = submetamatch.Groups(1).ToString().ToLower()) AndAlso (metaKey = [String].Empty) Then
                        ' if it's already set, HTTP-EQUIV takes precedence
                        metaKey = submetamatch.Groups(2).ToString()
                    End If
                    If "content" = submetamatch.Groups(1).ToString().ToLower() Then
                        metaValue = submetamatch.Groups(2).ToString()
                    End If
                Next
                Select Case metaKey.ToLower()
                    Case "keywords", "keyword"
                        fileKeyword = ReturnOutput(metaValue)
                        Exit Select
                    Case "description"
                        filedesc = ReturnOutput(metaValue)
                        Exit Select
                End Select
            Next

            

            '// ### Get the file SIZE ###
            filesize = filecontents.Length()

            '  Now remove HTML, convert to array, clean up words and index them ###
            'filecontents = stripHtml(filecontents)

            Dim wordsOnly As String = stripHtml(filecontents)

            '// ### If no META DESC, grab start of file text ###
            If (filedesc = "") Then

                If (wordsOnly.Length > 350) Then
                    filedesc = wordsOnly.Substring(0, 350) + "..."
                ElseIf (wordsOnly.Length > 100) Then
                    filedesc = wordsOnly.Substring(0, 100) + "..."
                Else
                    filedesc = wordsOnly + "..." '// file is only short!
                End If

            End If

            Dim SiteName As String = Replace(m_sSiteName, "'", "''")
            Dim SiteURL As String = Replace(m_sURLToProcess, "'", "''")

            Dim str1 As String = ""

            If fileKeyword = "" Then

                For i = 0 To m_sURLToProcess.Length() - 1

                    If m_sURLToProcess.IndexOf("/") > -1 Then

                        str1 = m_sURLToProcess.Replace("/", ",")

                    End If

                Next

                str1 = Replace(str1, ",,", "//")
                str1 += "," + m_sURLToProcess
                fileKeyword = str1

            End If

            
    
            Dim conn As New SqlConnection(strConn)
            Dim mySQL As String = "InsertPagesInfo"
            Dim cmd As New SqlCommand(mySQL, conn)
            cmd.CommandType = CommandType.StoredProcedure
            cmd.Parameters.Add(New SqlParameter("@SiteName", SiteName))
            cmd.Parameters.Add(New SqlParameter("@URL", SiteURL))
            cmd.Parameters.Add(New SqlParameter("@Title", filetitle))
            cmd.Parameters.Add(New SqlParameter("@Description", filedesc))
            cmd.Parameters.Add(New SqlParameter("@PageSize", filesize))
            cmd.Parameters.Add(New SqlParameter("@KeyWord", fileKeyword))

            conn.Open()
            iPageID = cmd.ExecuteNonQuery()
            conn.Close()

            Call ParseTxtLinks(filecontents, SiteURL, SiteName)


        Catch
            Debug.WriteLine("WriteToDB: " & m_sURLToProcess & ": " & Err.Number & ": " & Err.Description & "  " & Err.Erl)
            WriteErrorLog("WriteToDB: " & m_sURLToProcess & ": " & Err.Number & ": " & Err.Description & "  " & Err.Erl)

        End Try

    End Sub

#Region "Private Instance Fields"

    Private _goodUrls As New List(Of String)()
    Private _badUrls As New List(Of String)()
    Private _otherUrls As New List(Of String)()
    Private _externalUrls As New List(Of String)()
    Private _exceptions As New List(Of String)()

    Private _failedUrls As New List(Of String)()


#End Region

    '#Region "Public Properties"

    'Public Property GoodUrls() As List(Of String)
    '    Get
    '        Return _goodUrls
    '    End Get
    '    Set(ByVal value As List(Of String))
    '        _goodUrls = value
    '    End Set
    'End Property

    '    Public Property BadUrls() As List(Of String)
    '        Get
    '            Return _badUrls
    '        End Get
    '        Set(ByVal value As List(Of String))
    '            _badUrls = value
    '        End Set
    '    End Property

    '    Public Property OtherUrls() As List(Of String)
    '        Get
    '            Return _otherUrls
    '        End Get
    '        Set(ByVal value As List(Of String))
    '            _otherUrls = value
    '        End Set
    '    End Property

    '    Public Property ExternalUrls() As List(Of String)
    '        Get
    '            Return _externalUrls
    '        End Get
    '        Set(ByVal value As List(Of String))
    '            _externalUrls = value
    '        End Set
    '    End Property

    '    Public Property Exceptions() As List(Of String)
    '        Get
    '            Return _exceptions
    '        End Get
    '        Set(ByVal value As List(Of String))
    '            _exceptions = value
    '        End Set
    '    End Property

    '#End Region

    Public Sub ParseTxtLinks(ByVal filecontents As String, ByVal sourceUrl As String, ByVal SiteName As String)
        Const _LINK_REGEX As String = "href=""[a-zA-Z./:&\d_-]+"""
        Dim matches As MatchCollection = Regex.Matches(filecontents, _LINK_REGEX)

        For i As Integer = 0 To matches.Count - 1
            Dim anchorMatch As Match = matches(i)

            If anchorMatch.Value = [String].Empty Then
                _badUrls.Add("Blank url value on page " & sourceUrl)
                Continue For
            End If

            Dim foundHref As String = Nothing
            Try
                foundHref = anchorMatch.Value.Replace("href=""", "")
                foundHref = foundHref.Substring(0, foundHref.IndexOf(""""))
            Catch exc As Exception
                _exceptions.Add("Error parsing matched href: " & exc.Message)
            End Try


            If Not _goodUrls.Contains(foundHref) Then
                If IsExternalUrl(foundHref) Then
                    _externalUrls.Add(foundHref)

                    'MsgBox(foundHref)
                ElseIf Not IsAWebPage(foundHref) Then
                    foundHref = FixPath(sourceUrl, foundHref, sourceUrl)
                    _otherUrls.Add(foundHref)
                    'MsgBox(foundHref)
                Else
                    _goodUrls.Add(foundHref)

                End If
            End If
        Next

        'Good urls

        For Each link In _goodUrls

            Dim formattedLink1 As String = link
            Try

                Dim formattedLinkFinal As String = FixPath(sourceUrl, formattedLink1, sourceUrl)
                'MsgBox(formattedLink)

                Dim ipageId As Long = 0
                Dim filesize As Long = 0
                Dim filetitle As String = ""
                Dim filedesc As String = ""
                Dim filekeyword As String = ""

                'test codes

                Dim filecontents2 As String = GetWebText(formattedLinkFinal)

                Dim TitleMatch As Match = (Regex.Match(filecontents2, "<title>([^<]*)</title>", RegexOptions.IgnoreCase Or RegexOptions.Multiline))
                filetitle = TitleMatch.Groups(0).Value
                filetitle = ReturnOutput(filetitle) 'Remove Tag

                If filetitle = "" Then

                    filetitle = formattedLink1

                End If


                '// ### Parse out META data for description ###
                'Dim DescriptionMatch As Match
                'DescriptionMatch = (Regex.Match(filecontents2, "<META NAME=""DESCRIPTION"" CONTENT=""([^<]*)"">", RegexOptions.IgnoreCase Or RegexOptions.Multiline))
                'filedesc = DescriptionMatch.Groups(1).Value
                'filedesc = ReturnOutput(filedesc) 'Remove Tag


                Dim metaKey As String = [String].Empty, metaValue As String = [String].Empty
                For Each metamatch As Match In Regex.Matches(filecontents2, "<meta\s*(?:(?:\b(\w|-)+\b\s*(?:=\s*(?:""[^""]*""|'[^']*'|[^""'<> ]+)\s*)?)*)/?\s*>", RegexOptions.IgnoreCase Or RegexOptions.ExplicitCapture)
                    metaKey = [String].Empty
                    metaValue = [String].Empty
                    ' Loop through the attribute/value pairs inside the tag
                    For Each submetamatch As Match In Regex.Matches(metamatch.Value.ToString(), "(?<name>\b(\w|-)+\b)\s*=\s*(""(?<value>[^""]*)""|'(?<value>[^']*)'|(?<value>[^""'<> ]+)\s*)+", RegexOptions.IgnoreCase Or RegexOptions.ExplicitCapture)

                        If "http-equiv" = submetamatch.Groups(1).ToString().ToLower() Then
                            metaKey = submetamatch.Groups(2).ToString()
                        End If
                        If ("name" = submetamatch.Groups(1).ToString().ToLower()) AndAlso (metaKey = [String].Empty) Then
                            ' if it's already set, HTTP-EQUIV takes precedence
                            metaKey = submetamatch.Groups(2).ToString()
                        End If
                        If "content" = submetamatch.Groups(1).ToString().ToLower() Then
                            metaValue = submetamatch.Groups(2).ToString()
                        End If
                    Next
                    Select Case metaKey.ToLower()
                        Case "keywords", "keyword"
                            filekeyword = ReturnOutput(metaValue) + "..."
                            Exit Select
                        Case "description"
                            filedesc = ReturnOutput(metaValue) + "..."
                            Exit Select
                    End Select
                Next


                '// ### If no META DESC, grab start of file text ###
                If (filedesc = "") Then
                    Dim wordsOnly As String = stripHtml(filecontents2)
                    If (wordsOnly.Length > 350) Then
                        filedesc = wordsOnly.Substring(0, 350) + "..."
                    ElseIf (wordsOnly.Length > 100) Then
                        filedesc = wordsOnly.Substring(0, 100) + "..."
                    Else
                        filedesc = wordsOnly + "..." '// file is only short!
                    End If

                End If


                Dim str1 As String = ""

                If filekeyword = "" And filetitle.IndexOf(".txt") > -1 Then

                    For i = 0 To formattedLinkFinal.Length() - 1

                        If formattedLinkFinal.IndexOf("/") > -1 Then

                            str1 = formattedLinkFinal.Replace("/", ",")

                        End If

                    Next

                    str1 = Replace(str1, ",,", "//")
                    str1 += "," + formattedLinkFinal
                    filekeyword = str1

                End If

                


                filesize = filecontents2.Length()

                Dim conn As New SqlConnection(strConn)
                Dim mySQL As String = "InsertPagesInfo"
                Dim cmd As New SqlCommand(mySQL, conn)
                cmd.CommandType = CommandType.StoredProcedure
                cmd.Parameters.Add(New SqlParameter("@SiteName", SiteName))
                cmd.Parameters.Add(New SqlParameter("@URL", formattedLinkFinal))
                cmd.Parameters.Add(New SqlParameter("@Title", filetitle))
                cmd.Parameters.Add(New SqlParameter("@Description", filedesc))
                cmd.Parameters.Add(New SqlParameter("@PageSize", filesize))
                cmd.Parameters.Add(New SqlParameter("@KeyWord", filekeyword))

                conn.Open()
                ipageId = cmd.ExecuteNonQuery()
                conn.Close()

                'If formattedLink <> [String].Empty Then
                '    'CrawlPage(formattedLink)
                'End If
            Catch exc As Exception 'to debug later on
                _failedUrls.Add(((formattedLink1 & " (on page at url ") + sourceUrl & ") - ") + exc.Message)
            End Try

        Next

        'External------------------------------------------------------------------

        For Each link In _externalUrls


            Dim formattedLink1 As String = link
            Try

                'Dim formattedLinkFinal As String = FixPath(sourceUrl, formattedLink1, sourceUrl)
                'MsgBox(formattedLink)

                Dim ipageId As Long = 0
                Dim filesize As Long = 0
                Dim filetitle As String = ""
                Dim filedesc As String = ""
                Dim filekeyword As String = ""

                'test codes

                Dim filecontents2 As String = GetWebText(formattedLink1)

                Dim TitleMatch As Match = (Regex.Match(filecontents2, "<title>([^<]*)</title>", RegexOptions.IgnoreCase Or RegexOptions.Multiline))
                filetitle = TitleMatch.Groups(0).Value
                filetitle = ReturnOutput(filetitle) 'Remove Tag

                If filetitle = "" Then

                    filetitle = formattedLink1

                End If


                ''// ### Parse out META data for description ###
                'Dim DescriptionMatch As Match
                'DescriptionMatch = (Regex.Match(filecontents2, "<META NAME=""DESCRIPTION"" CONTENT=""([^<]*)"">", RegexOptions.IgnoreCase Or RegexOptions.Multiline))
                'filedesc = DescriptionMatch.Groups(1).Value
                'filedesc = ReturnOutput(filedesc) 'Remove Tag


                Dim metaKey As String = [String].Empty, metaValue As String = [String].Empty
                For Each metamatch As Match In Regex.Matches(filecontents2, "<meta\s*(?:(?:\b(\w|-)+\b\s*(?:=\s*(?:""[^""]*""|'[^']*'|[^""'<> ]+)\s*)?)*)/?\s*>", RegexOptions.IgnoreCase Or RegexOptions.ExplicitCapture)
                    metaKey = [String].Empty
                    metaValue = [String].Empty
                    ' Loop through the attribute/value pairs inside the tag
                    For Each submetamatch As Match In Regex.Matches(metamatch.Value.ToString(), "(?<name>\b(\w|-)+\b)\s*=\s*(""(?<value>[^""]*)""|'(?<value>[^']*)'|(?<value>[^""'<> ]+)\s*)+", RegexOptions.IgnoreCase Or RegexOptions.ExplicitCapture)

                        If "http-equiv" = submetamatch.Groups(1).ToString().ToLower() Then
                            metaKey = submetamatch.Groups(2).ToString()
                        End If
                        If ("name" = submetamatch.Groups(1).ToString().ToLower()) AndAlso (metaKey = [String].Empty) Then
                            ' if it's already set, HTTP-EQUIV takes precedence
                            metaKey = submetamatch.Groups(2).ToString()
                        End If
                        If "content" = submetamatch.Groups(1).ToString().ToLower() Then
                            metaValue = submetamatch.Groups(2).ToString()
                        End If
                    Next
                    Select Case metaKey.ToLower()
                        Case "keywords", "keyword"
                            filekeyword = ReturnOutput(metaValue) + "..."
                            Exit Select
                        Case "description"
                            filedesc = ReturnOutput(metaValue) + "..."
                            Exit Select
                    End Select
                Next

                '// ### If no META DESC, grab start of file text ###
                If (filedesc = "") Then
                    Dim wordsOnly As String = stripHtml(filecontents2)
                    If (wordsOnly.Length > 350) Then
                        filedesc = wordsOnly.Substring(0, 350) + "..."
                    ElseIf (wordsOnly.Length > 100) Then
                        filedesc = wordsOnly.Substring(0, 100) + "..."
                    Else
                        filedesc = wordsOnly + "..." '// file is only short!
                    End If

                End If


                Dim str1 As String = ""

                If filekeyword = "" Then

                    For i = 0 To formattedLink1.Length() - 1

                        If formattedLink1.IndexOf("/") > -1 Then

                            str1 = formattedLink1.Replace("/", ",")

                        End If

                    Next


                    str1 = Replace(str1, ",,", "//")
                    str1 += "," + formattedLink1
                    filekeyword = str1


                End If


                filesize = filecontents2.Length()

                Dim conn As New SqlConnection(strConn)
                Dim mySQL As String = "InsertPagesInfo"
                Dim cmd As New SqlCommand(mySQL, conn)
                cmd.CommandType = CommandType.StoredProcedure
                cmd.Parameters.Add(New SqlParameter("@SiteName", SiteName))
                cmd.Parameters.Add(New SqlParameter("@URL", formattedLink1))
                cmd.Parameters.Add(New SqlParameter("@Title", filetitle))
                cmd.Parameters.Add(New SqlParameter("@Description", filedesc))
                cmd.Parameters.Add(New SqlParameter("@PageSize", filesize))
                cmd.Parameters.Add(New SqlParameter("@KeyWord", filekeyword))

                conn.Open()
                ipageId = cmd.ExecuteNonQuery()
                conn.Close()

            Catch exc As Exception
                _failedUrls.Add(((formattedLink1 & " (on page at url ") + sourceUrl & ") - ") + exc.Message)
            End Try

        Next

    End Sub

    Public Function FixPath(ByVal originatingUrl As String, ByVal link As String, ByVal siteUrl As String) As String
        Dim formattedLink As String = [String].Empty

        If link.IndexOf("../") > -1 Then
            formattedLink = ResolveRelativePaths(link, originatingUrl)
        ElseIf originatingUrl.IndexOf(siteUrl.ToString()) > -1 AndAlso link.IndexOf(siteUrl.ToString()) = -1 Then
            formattedLink = originatingUrl.Substring(0, originatingUrl.LastIndexOf("/") + 1) + link
        ElseIf link.IndexOf(siteUrl.ToString()) = -1 Then
            formattedLink = siteUrl.ToString() + link
        End If

        Return formattedLink
    End Function

    Private Function ResolveRelativePaths(ByVal relativeUrl As String, ByVal originatingUrl As String) As String
        Dim resolvedUrl As String = [String].Empty

        Dim relativeUrlArray As String() = relativeUrl.Split(New Char() {"/"c}, StringSplitOptions.RemoveEmptyEntries)
        Dim originatingUrlElements As String() = originatingUrl.Split(New Char() {"/"c}, StringSplitOptions.RemoveEmptyEntries)
        Dim indexOfFirstNonRelativePathElement As Integer = 0
        For i As Integer = 0 To relativeUrlArray.Length - 1
            If relativeUrlArray(i) <> ".." Then
                indexOfFirstNonRelativePathElement = i
                Exit For
            End If
        Next

        Dim countOfOriginatingUrlElementsToUse As Integer = originatingUrlElements.Length - indexOfFirstNonRelativePathElement - 1
        For i As Integer = 0 To countOfOriginatingUrlElementsToUse - 1
            If originatingUrlElements(i) = "http:" OrElse originatingUrlElements(i) = "https:" Then
                resolvedUrl += originatingUrlElements(i) & "//"
            Else
                resolvedUrl += originatingUrlElements(i) & "/"
            End If
        Next

        For i As Integer = 0 To relativeUrlArray.Length - 1
            If i >= indexOfFirstNonRelativePathElement Then
                resolvedUrl += relativeUrlArray(i)

                If i < relativeUrlArray.Length - 1 Then
                    resolvedUrl += "/"
                End If
            End If
        Next

        Return resolvedUrl
    End Function

    Private Function IsAWebPage(ByVal foundHref As String) As Boolean
        If foundHref.IndexOf("javascript:") = 0 Then
            Return False
        End If

        Dim extension As String = foundHref.Substring(foundHref.LastIndexOf(".") + 1, foundHref.Length - foundHref.LastIndexOf(".") - 1)
        Select Case extension
            Case "jpg", "css", "pdf"
                Return False
            Case Else
                Return True

        End Select
    End Function

    Private Function IsExternalUrl(ByVal url As String) As Boolean
        If url.IndexOf("localhost") > -1 Then
            Return False
        ElseIf url.Substring(0, 7) = "http://" OrElse url.Substring(0, 3) = "www" OrElse url.Substring(0, 7) = "https://" Then
            Return True
        End If

        Return False
    End Function

    Private Function GetWebText(ByVal url As String) As String
        Dim request As HttpWebRequest = DirectCast(HttpWebRequest.Create(url), HttpWebRequest)
        request.UserAgent = "A .NET Web Crawler"

        Dim response As WebResponse = request.GetResponse()

        Dim stream As Stream = response.GetResponseStream()

        Dim reader As New StreamReader(stream)
        Dim htmlText As String = reader.ReadToEnd()
        Return htmlText
    End Function

    Private Function stripHtml(ByVal strHtml As String) As String

        'Strips the HTML tags from strHTML

        Dim objRegExp As System.Text.RegularExpressions.Regex = New System.Text.RegularExpressions.Regex("<(.|\n)+?>")

        ' Replace all tags with a space, otherwise words either side
        ' of a tag might be concatenated
        Dim strOutput As String = objRegExp.Replace(strHtml, " ")

        ' Replace all < and > with &lt; and &gt;
        strOutput = strOutput.Replace("<", "&lt;")
        strOutput = strOutput.Replace(">", "&gt;")

        Return strOutput

    End Function

    Private Function ReturnOutput(ByVal Input As String) As String
        Dim bInsideTag As Boolean = False               'Check to see if we're inside a tag or not
        Dim strOutput As String = ""
        Dim sOut As String = ""

        'Go through the string, and pull out all the tags
        For i = 1 To Len(Input)
            Select Case Mid(Input, i, 1)
                Case "<"
                    bInsideTag = True
                Case ">"
                    bInsideTag = False
                Case Else
                    If Not bInsideTag Then sOut = sOut & Mid(Input, i, 1)
            End Select
        Next i

        Return sOut

    End Function

End Class
